{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'1.14.0'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-3-c974fcd23729>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting .\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting .\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting .\\t10k-images-idx3-ubyte.gz\n",
      "Extracting .\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From D:\\ProgramData\\Anaconda3\\envs\\tf1.14\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "Extracting .\\train-images-idx3-ubyte.gz\n",
      "Extracting .\\train-labels-idx1-ubyte.gz\n",
      "Extracting .\\t10k-images-idx3-ubyte.gz\n",
      "Extracting .\\t10k-labels-idx1-ubyte.gz\n",
      "[[0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]\n",
      " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]]\n",
      "[7 3]\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('.',one_hot=True)\n",
    "mnist2 = input_data.read_data_sets('.',one_hot=False)\n",
    "print(mnist.train.labels[:2])\n",
    "print(mnist2.train.labels[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55000, 784)\n",
      "(55000, 10)\n",
      "(5000, 784)\n",
      "(5000, 10)\n",
      "(10000, 784)\n",
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "print(mnist.train.images.shape)\n",
    "print(mnist.train.labels.shape)\n",
    "\n",
    "print(mnist.validation.images.shape)\n",
    "print(mnist.validation.labels.shape)\n",
    "\n",
    "print(mnist.test.images.shape)\n",
    "print(mnist.test.labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAI+CAYAAABe7hvVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3xUVf7/8c9JIZSErii9hmYBFXtBAQti710XRcG69rJ+XdZd3bUCYqPYe1lUbFjBhoKI0jsoHek9pJzfHwm/5XPPMJNJZjIzZ17Px8PH7vvk3DtHuUw+ufPJucZaKwAAAL7JSPQCAAAA4oEiBwAAeIkiBwAAeIkiBwAAeIkiBwAAeIkiBwAAeIkiBwAAeCmtihxjjDXGbDHG/Kuc8/saYzaXHdc23utD8uMaQixU4DoaWDbfGmOy4r0+JD/ei8rHpNNmgMYYKyLtrLXzyvJRIvJJYFotETnbWvvu7o5D+gpxDTUUkfdFpIOIZIrITBG51Vr7fbjjkN7CXQ/GmMtE5AURucpaO2KX8ZYislBEsq21RVWzUiSrUNeQMSZTRAaKyF9EJE9E5onIsdba9eGO81la/0Rgrf1WRHJ3ZmNMdxEZLSKfJmpNSDmbpfQNZa6IWBE5TURGG2P25BsRomWMqScid4nI9ESvBSlpoIgcLiKHicgfItJZRLYndEUJllYfV5XDZSLyjrV2S6IXgtRgrd1urZ1trS0RESMixSJST0TqJ3ZlSFEPisgQEVmd6IUgtZQVyDdJ6R3A322padZaihyIGGNqisjZIvJioteC1GOMmSKlPzF9ICIjrLWrErwkpBhjzMEicpCIPJPotSAl7SsiRSJytjFmhTFmjjHm2kQvKtHS+uOqgLOk9KencYleCFKPtXY/Y0x1ETlDRKolej1ILWW9FE+JyPXW2hJjTKKXhNTTVETqiEi+iLQSkXYi8qUxZo619vOEriyBuJPzP5eJyEs2nTqxEVNlH129LiJ3GmP2T/R6kFIGiMgUa+34RC8EKWtb2f/+w1q7zVo7RUTeEJHeCVxTwlHkiIgxppmIdBeRlxK8FPghW0RaJ3oRSCk9ROSMso8ZVkhp8+ijxpihCV4XUseUsv/lB/Vd8HFVqUtE5Adr7fxELwSpxRhzqJT+PZogpb9CfoOINBKRnxK5LqScy0Wk+i75vyLyjoiMTMhqkHKstfONMd+KyD3GmBuk9Aet80TkgsSuLLEockpdKiIPJ3oRSEk5UvrbMK1FpFBEporIydbaZQldFVLKrvuYiIgYY3aIyEZr7YYELQmp6QIpLYzXiMgqEbnXWvtlYpeUWOn2cVWBiEwyxty/66C1toO11vmJyRhzhTFmfdlxJVW0RiQ3dQ1Za8dZa/e31uZZa+tba4+x1n6zczLXEHYj5HvRTtba7oGNAO8Tkd/KjuPjCIiEuIastUuttSdaa3Otta2ttc/u/Fq6vhel1Y7HAAAgfaTbnRwAAJAmKHIAAICXKHIAAICXwv52Va+Mc2jY8dTnJW9X2ZaqXEf+qqrriGvIX7wXIRZ2dx1xJwcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHgp7LOrAACAJzIyVZwzvKvK0094SuVTLuvvnCLry0mxX1cccScHAAB4iSIHAAB4iSIHAAB4iZ4cAAA8k9WimTM258EGKi/sPiIwo5pK69tUk6CGX1Z6aVWKOzkAAMBLFDkAAMBLFDkAAMBL9OQAlZDZKV/lWf3rOXPmnvm0yiViVc4Qo/JT61s553jxsd4qNxg5Pqp1AvBbVuuWKs+4p6Ezx+3B0a5afITKjb5d7cwpjn5pCcWdHAAA4CWKHAAA4CWKHAAA4CV6coAwspo1VXnGfXup/Ppxz6rcNafEOUdJ4GeJEgnO0V/vV3eec47Gd7yq8nNjjlK5aMlS5xhUvYzq1Z2x5t8Eeq6afK9yptF//jN3bHXOccsJl6pcPNu9RpBeTLbew2bm3+urvLBn+P4bEZHWX/xF5fb9Zqhcsn1uBVeXPLiTAwAAvESRAwAAvESRAwAAvESRAwAAvORt4/Hymw9X2ej916T6Gj2wroN7jr3H622Pqo+eEJO1ITkteOgwZ2zWRU+qHGkjv2CTsYjIR1vrqDxhc+uw6ziw1iJn7KzcjSovGzNN5Q87u5sQIv6CjcZL33A3cvywyavO2K66TztdZfOou4lbzvxfK7C68LJaNle5aNEfMX8NxM/sofurvLDn8IjHtB17ucrtLv1FZffXJlIfd3IAAICXKHIAAICXKHIAAICXYt6Ts+pa3Quzfr9ClUcdPzTWLxlSx2oTw359uy1SuU5GDWfOqku2qLxsiP7P9diKXiqvObe2c46ixUvCrgPJ45xe3ztjwR6cSBv5Pbm+jXOOz0/orHKkjfu+P+V8Z+zUZ/RDPoMbBn4o3cKeE/Ex7+9dVZ7V7cndzPyfdl9eqXL7/rNVLtmyyDnGOiPRmzNMXyPvH/+Eyue9cLPKzf/+QwxeFbEyb9ChOvd5KjBDvxe1/vwvEpTfb7rKsbiukh13cgAAgJcocgAAgJcocgAAgJcq1ZMzZ7jbBzCr92CVc0x2YEZOZV4yZtx1ufbMrBXI+usvtfhG5Yvf7O6cY92F7EWRtA7eV8VrGjztTPloq34gZ3CPm2kbG6tccNsezjnmP6QvnPz7a6pcPFM/BC/UfkzZz+pzFAY+TF96h+6Fa/If+iniwR6m9yb55sKHAzNqStAfRfqBm/l99R5HJYU7YrK2XRX2PNAZG9VL90N2DjzgEcljx4nu99ZRpw9SOdPoPZqcPXCu+M05hy0pdsZ8x50cAADgJYocAADgJYocAADgpUr15Dx97EvOWLDX5T9r2qm8akdeZV5SRET+O0l/3tx8tNnNzMpZ0kPXgA/1fk3l4POEXmk51jnHxa91V3ndeU1VZh+dBJowVcV+Z/V3pmQuX6uyu8fNCpWW3uE+l2rmMXo/kpOGX6VfY6aev6av+wytQjtJ5eB+PS1e/V2v0zkDYmHlHbp/Zs9M3YOzzbr9NZfedIvKNQt/iv3CAjb/daMztm81/d682Rao3OrtNSqnX/dG8mhwz0JnbL9qugen18xTVM6/T/+ZF6dh/00o3MkBAABeosgBAABeosgBAABeosgBAABeqlTj8aDzznbG/tZFP6Ryz/f0w+eK1+hGzorIl/AP34yVtqN1HvFcb5VXvKEf6Hht3cXOOYLNyO376ebWlvfSeJws7MSpzli0DbzVV7uPvBu2oaXK1VZuVnnBQL2R3wuX6EZlEZEM0c31kwr0zyeRHvqJ2OiX/13Yr58x+xxnrOao8I3GJku/DZsa7sOCIyneVze8P97x+YjHdJ90hcp7Tp8V9esiPm5o8kXEORtf1L/EUnfu+HgtJ6VxJwcAAHiJIgcAAHiJIgcAAHipUj05dtJ0Z6yB3rPMqw2lSqboz6yff7yPytcOdB/wGPTaxfoBpnffe3DlF4a42Xaa/vNZ20H/lQn24DSYqvttRET61VmkcpcP9cZ9B+focwQ3+hMRmRjowflb38CGgvKLcwyqXl72dmdsSyAXHn+QyvXvXaTym60/q8Arj4s44/vANbTHv5PjYckQ2XDxoSofXf1XZ84RU85Uue7LP8Z1Tb7gTg4AAPASRQ4AAPASRQ4AAPBSpXpyAN8tO08/cHHmMbrvKrh/TYm4++QE5wR7cCLtgSMicsk716nc+mv2xEiEYcP1QxGvuXWoyi+1DmyuJSLX/HCiyiNb6GsoSzJjtLrwLh99jcrtxtPTkSzWnxrs3HJt/WAvlXPtgngtJ7yMwPWa5A8C5U4OAADwEkUOAADwEkUOAADwEj05UVhyt37GUEnXTVGfo1Gm7vEoOu5AlbO+Cmw0hKTi7mGTEeHrkef0W3ycyovvauecgR6c5LClaag/3/+pYao5Yy+2+Cowonsablmh92L6eEw35xyFe+v3jXnHDw+7jlAa/mIiT0JC7F1vY8Q5NdaEv/ZioeAkfe2tvmqrM2efRstV3nS2vuaLlq+I/cIqgTs5AADASxQ5AADASxQ5AADASxQ5AADAS942Hme1bqnyvL57q/zU+cOiPmf36vohiJkm+hqxaVauysOe1w/sHNDiyKjPifhp/KZuqjunid4Mbp/ay1S+psEPzjmaZNYMjOjrZv6DHVWu8fWEKFeJqpL/7J8qdyy8NupztH15rcols+er3KrIbTJf8O/DonqNAUuPcMbqv6Z/qcHdthJVJWuvRioPb/9qYEauxENm3Toqnz5+rsrn5Q1RuU5GjYjn7Dz0IpWbnkXjMQAAQNxR5AAAAC9R5AAAAC+lZE/O5nMOUfnPA9xa7R9nvqHy+XnrYvDKsa8Je35xk8r58nPMXwMVV+N93R9T8L7++qTANdGvW3/nHJvu1w/f+2rfN1U+8u/6QYm/TWrmnKNoydKIa0X8Fc8J9M/cOX83M8OcowKvm7U1uo38fh7RxRlrWMiGkkkjO1vF5lmx78FZNeBwZ+z0q8eq3K/OssCMyD04QXvkRX64aCJxJwcAAHiJIgcAAHiJIgcAAHgpKXtyTNfOKtcdqh8I9nHLp1WuyH41723Rn4FO29Y04jEfPtRdv26B3mnisn+Mdo5xP/PUqq3IDvt1lE9WM/fPr2jxkipfh5041RnLPVHnc8bpvXZGtf1Y5X2udPdKav53enLSmYnQyFMU6PSpN6cgjqtBZdlN+uHOwzY0VjnS9w0RkcyGDVRe/Jf2Kk+96akKri46G7ZVV3nPKnnV8uNODgAA8BJFDgAA8BJFDgAA8FLCe3J+H+j+Lv+95+t9RC7KW6PyH0VbVZ61o55zjutfv1Llmsv1PhN7j12tcvGMORHXWkd+DPv1uXc1cgcDn60uLNyscsv3dUb5bDvtYJWDe82IiHz4u+7t2vv0mXFdU3lteKS5yiXP6N6uwnbbqnI5SAFXXDAm7NfPmaf7vDLH/rKbmUgGxes3qPz6km4q96sT2JBLRI644yeVu92/QOVzc7+M0erCG/hnJ5Ub36D3ySmqklWUH3dyAACAlyhyAACAlyhyAACAlyhyAACAlxLeeFy32ypnLNho3GPGqSoXPrGXysGHKIqItJTwD6OryEPygkqO6ary6XVHhpil68i1JdX0lye4m8fBFdzs77wHP1H5540tnWOSpdE4s24dlc/+t24izZDoHr4Iv2XusYcz1i5nXthjVj/dUuU8WRHLJSHOtj+/t8oFDxc6cx7ea3Lc11Fo9XfGTuP6OnPy79Lfn4t+XxzXNVUWd3IAAICXKHIAAICXKHIAAICXEt6T06Cvuxle25v7q9zmNt1fkyV/xHVN5bUuXz+Y7IjqkWvGftMuVrmhRN6EECK/X6g30AtulvX45J7OMW0k/p9hOw7e1xk66flvVO5XV/dXlAR+1sieUyP260LK2HBsG2fslJq6j2uz1Q/grL7a7eFA6qj9mt7M9Kd/ug9uPrq6MxS1Ylui8kE/X6hytXf0xrqtX3Z7W5Nts79IuJMDAAC8RJEDAAC8RJEDAAC8lPCenKLl7n4ObW5LjT0e1nSL/OnkzB36YaJ5T9XZzUyE0+TrTSpn35ip8o1dvnKOGXn9ySo3mK77GLK+mhTxdTM75au8rEdDlXNP1tfq1/u+4JwjuA9OsAcn/5OrdR74Q8R1wV+XDfwg4pyFhYE+ri8iX8vwS4fvLlHZTMtTudWQ6c4xtlj35Oy5aVbsF5ZkuJMDAAC8RJEDAAC8RJEDAAC8lPCenFRywrSNKo+q+2RgRuC5VCJy2fTLVK73ycRYLys9BJ7xdcSUM1X+at83nUOuufMJlUtEfx49cNWBEV/21Dqvq9w1R58jI/BzQvA1ds7aVft3rlW508P62S+ptg8FYqtBprt3WNAjy08IjKyPz2KQNDo9PUDllg/qZzbaIv3OEYvnM/qAOzkAAMBLFDkAAMBLFDkAAMBLFDkAAMBLNB5H4ezaU1SumZGr8pzCLc4xNYfWjeua0lXdq3aoPPADt4n4gUb6z6vQ6q/fv+evKpdIYIJE3shvZfE2lZ9ac7hzjs+GHqFyu5H6oXc0GiNaO0oyI09CyvpX6y7OWDPRm4S671YIhTs5AADASxQ5AADASxQ5AADAS/TkhLFqgO6vaJSpN/JbWKg37brggducczT8ZLwzhsorWrxE5d9OaebMafuf8Jv9zew+QuWjp5zrzPlzbe2w52g7SHfU2IlTnTkNhGsAsTW85YcqH/joX1Vuc8uPVbkcIGlxJwcAAHiJIgcAAHiJIgcAAHiJnpwyJifHGTvrmq9U3lSi92bpPaG/ys2fpfciUYqWLHXG2lzkju2qj+iendoy35kTviOHvSoQe/e8cZEz1uHSx3TODrxflej9nACU4k4OAADwEkUOAADwEkUOAADwEj05O5W43RUvjz5W5U9+665y87fYiwJAbLX4P7e37+b/OyzsMW3YiwkIiTs5AADASxQ5AADASxQ5AADASxQ5AADASzQel7GFO5yxlvfQzAcAQKriTg4AAPASRQ4AAPASRQ4AAPCSsZZHDAIAAP9wJwcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHiJIgcAAHgprYocY4w1xmwxxvyrnPP7GmM2lx3XNt7rQ/KrwDXUs+waKjHG9Iz3+pAaeC9CZVXgGhpYNt8aY9Lm4dxpVeSU2d9ae8/OYIwZZoyZXfZN6PJdJ1prR1prc6t8hUh2wWvoOGPML8aYjcaYBcaYfju/Zq39ouwa+iMhK0Uy470IlRW8hroYYyYZY7aW/W+XnV+z1t4nIp0TssoESsciJ+g3ERkgIr8keiFIPcaYbBEZJSLPikgdETlPRB4zxuyf0IUhFfFehAozxlQTkfdF5BURqSciL4rI+2XjaSvtixxr7ZPW2i9FZHui14KUVF9EaovIy7bURBGZKSKdErsspBrei1BJ3UUkS0QGWWsLrLVDRMSIyHEJXVWCpX2RA1SGtXaliLwuIlcYYzKNMYeJSAsR+S6xKwOQZjqLyBSrn7o9RdLwI6pdpU3zERBHr4vICBEZXJb7W2sXJ3A9ANJProhsCIxtEJG8BKwlaXAnB6gEY0wHEXlTRC4VkWpS+lPT7caYkxO6MADpZrOUfnS+q9oisikBa0kaFDlA5ewjIrOttWOstSXW2tki8pGInJTgdQFIL9NFZD9jjNllbL+y8bSV9kWOMaaaMaa6lDZoZRtjqhtj0v6/C8ptsoi0K/s1cmOMaSMifaT0N2WAcuO9CJU0VkSKReQGY0yOMea6svGvErekxOMvkMhnIrJNRA4XkWFl///ohK4IKcNaO19E/iIiQ0Rko4iME5F3RWRkIteFlMR7ESrMWrtDRE6X0o/O10vp+9LpZeNpK92KnAIRmWSMuX/ngLW2u7XWBP4ZKyJijLnCGLO+7LiSxCwZSSbUNfSWtXYfa22etbaptfYOa22JiIgxpkfZNdRISn/KAkR4L0LlhbqGJltrD7TW1rDWHmCtnbzza8aY+6T0DnOBiFj3dH4y+rfNAAAA/JBud3IAAECaoMgBAABeosgBAABeCrvjca+Mc2jY8dTnJW+byLNig+vIX1V1HXEN+Yv3IsTC7q4j7uQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvUeQAAAAvZSV6Acli4Rv7OWPfHfG0yhdeer3KmV//Etc1AYCIyPxHD1X5xhM/UfnjCw5TuWTKrLivCUnmUP09bOGNxpky55gXVW479nKV21z4a8yXlWjcyQEAAF6iyAEAAF6iyAEAAF6iJ6eM/aOWM9bgqBoqr22fo/IeX8d1SfBEwcndVF571WaVJ3d7NepzXrPkKJW/+2R/lVs/u8A5pmj5iqhfB1Uvq0ljZ2zoac+r3KvGNpVfPKS3yg2mxH5dSC4rbjpc5Qeue07l42tscY4ptDoPPvgNlYdIh4ivu/J6/bqNX9P9X8Vr1kY8R1XiTg4AAPASRQ4AAPASRQ4AAPASPTllai1x9xQI2uu831UufiZeq0GqMNnVVJ7zWFdnzkenPK5y22zd21VSgdd9pum3+hxXfaNyl30vdY5pehY9Oalg/tUtnLFgDw78ZnJynLF15x6g8je3PqpyTaPfi2JhyV2HO2MTrx2k8lvXNlV5yKCzVN7jmfExX1c0uJMDAAC8RJEDAAC8RJEDAAC8RJEDAAC8RONxFLYVZasc+zYvpJrZT3RRec4pTzlzMqS6yiVinTmR9FvcXeURzcaFnT+kyxvO2KMNjlE52TbtQqlmRyxJ9BKQYAv+foAzNv3SoYGR6L8DPbO+tcrPvnyyyk3kB5ULGri/FpFtMlW+KG+5yt3ufEzlS+Rm5xxV2YzMnRwAAOAlihwAAOAlihwAAOAlenLK1D55ecQ5G97VD87bQ37fzUz4IrjZX7AHZ3qf4OfkmRK0vHirykePulXl1qN2qJwz1920r3j1GpW7vnmRypO6vaLyL9taOuewOwqdMSTe9j4Hqzy49RMhZmWHGIMvgpv/1eq0rtLn/GRrnjP27u3Hq9zkox+cOZWVH3jPfOOuR5w5J3S9SR9z9cSYr2Mn7uQAAAAvUeQAAAAvUeQAAAAvpW1PTnF3vQ/B6M5POnN+3aH7Kxq9Ok3lijxYEall+bUHqTznlGC/hL5GRm5o7pzjv1f1Urnd9z+Gfc2icqyroCB8j8bopfs5YzU2LSzHmVHVtjXQ19C+1ei/8Z3J0t965/9Dfz+acVCw1y+y4F5aq85ye3JylkbX+9Lyox3O2H4tLld50mEjVQ7uo9MqS+8TJiJSe1bVXePcyQEAAF6iyAEAAF6iyAEAAF5K356cHF3f5ZocZ06h1c8YKtm0Ka5rQvLp3+99lTPEqPzgmk4qjz813zmHWfRrVK+ZWbu2M7bkyn1Uvn2//6o8eYfuEKtxAv03Pvu+QL9/5S0uTycXkkVBz64qz7g4+h6cG5cdofLKk3WfS/GaZdEvLCDz61+cseZf6zxq9t4qn5u7qtKvG0vcyQEAAF6iyAEAAF6iyAEAAF6iyAEAAF5K28bjRWdQ3yGy4sDPASWim9E/fqC7ynmLwm/0JyIiGXqzrOJj9le5z9AvnUOuqau7/YIN0CfPPj1wxNLI60BS6HDN9KiPGbREbzBZ7dP4PeAQlbfyhsNVHtD/vajPEWw0XnhM4L1p69roF5YG+E4PAAC8RJEDAAC8RJEDAAC8lLY9OXl7sbEfKq/mCvcBdpEEe3A+eWV41Oc4Y15vlTPO2qpfI+ozIlEGNArsrhbotwpl9iftVG4qf8ZwRaiMjP07OmP/vkE/xLJHja3OnF0FH7Yp4m72l6geHNO1s8ots90NA3c1r7DAGauzoOo2r+RODgAA8BJFDgAA8BJFDgAA8FLa9uQA5TF3WyM9UGeRis+9NETlf6/s6Zxj7O9tVf704CGBGTVU2lCy3TlHt4/+qnKHW/TeKiVbtjjHwF8t3tM9OPRgJY+jXnZ7VCL14ARNfG9fZ6zJmh8qvKZYmt2/psoH59jdzCw1ZksnZ6zG+xNiuqZwuJMDAAC8RJEDAAC8RJEDAAC8lDY9ORnVq6t8ZJOFEY8ZvuqYwMjmGK4IqWDmtYHPk9/9ScW9M3U/zeDG3zvnyGisP0svCfTgBB37xG3OWP5DwXMgVQWfY9Q+O9hrUV2ClhYHejqK6MJJFquvPkzl/vUeDTErR6XlxdtUvvl3/ey55v9d6ZwhEX/iWa1aOGPjTnw8MBL+/ey7tW1DjK6u+KKixJ0cAADgJYocAADgJYocAADgJYocAADgpfRpPK5bR+UnGn8S8Zhx3+2jchv5MaZrQnIpOLmbM7b4fP0guYxyPDwxKNMEfpawum24x/QzVW78UHJs+oXYyGy0p8pdL5yqcu0Mt9E4qPuoW1VuN5f3omSxKdCbm5uRE3riLh5Zdaw+x1HBRtyqa8wNZ/a1eztjwV+2CFoX2Mx0xeA2zpxaNB4DAABUDkUOAADwEkUOAADwUtr05BS1bBR5UkDzTwvjsBIkSsZ+HVTea9hSlUc0e9Y5pkRsIId35wq3r+e/Ew5S+eleL6o8sv0rKl96ru6/EBHJfYsejJTVsJ6KI5p9Gnb6xhAPaM1byM+jPvn0C/2e0ErGJ2glAUb3HNrM6E9x65KTVK71zk+7mVk1+JsDAAC8RJEDAAC8RJEDAAC8lDY9OavvcT/n3lXvWac6Y9XG/qaydWYgma3upx+cN+beR1Su4+xPEnkPnFuWH6ryJ1/pz9bzH3cf/Jq/fILKjxx7kT7HK8NVPv8+dw+nD9+q54whNRTXqhbV/KmFNZ2xvQaxd5JP9v4+OR+wuuGiQ1Sede6TUZ/jh+/1Q40Tvb8cd3IAAICXKHIAAICXKHIAAICX0qYn5+l9Xg2M6A0Alm2s7RzTuGhJHFeEWNp0/qHOWKQenJmFeh+kx1f0cs4xe1BnfY73flW59Xa9v4V+0lVomeN0r1eHt65V+bdzBjnHjDr+OpWzP/u5HK+EZJD36PKo5veffJEz1lSmx2o5SAIt7p6l8srRVfO6WU2bqDz32uYq/3Txo4EjIj+H6/VNeg+6/OfXqZzo7iPu5AAAAC9R5AAAAC9R5AAAAC9R5AAAAC9523ic1VI3VOUZvZlWpsmuyuUgzlbv527kF2w0HrWlvsrPn3uyyiW/znDOkRfYyCrSAzrLI6OGXlfnAxapnBPi2izJirxRIRIvq1lTZyw/94+wx1y0qKfKLa5c5sxJdPMmYuvIuvNUfq+d/sWJ4rkLoj5nZsd2Ks+9rKEzZ9DZz6t8fI0tgRmRG42DXrz2NJWzpk+K+hzxxJ0cAADgJYocAADgJYocAADgJW97craP0Dk/W/dBFFvdXZH7lrsZIFJbRuCBm3d8fa7K+b9OrJJ1ZDZsoHLNUXpdb7b+OHAE/TepakXvZs7YB3t+oHKm0T9brtuuH8iZsUNvpiYiYrL1Qz5t4Y6KLhEx1m6E3uxxYO8uzpz79tCbiF5Re7HKmR/o7x+1kgsAACAASURBVEdTt7q9XZF0qTVO5YvyotuEMpQPtrgPBr71i/NV7vCj3qgyFn2LscSdHAAA4CWKHAAA4CWKHAAA4CUvenIy89s4Y7e0/CDEzP+5YKF+GGPtN36K6ZpQtRpOsc7YupJtKk/srR982e3Zm1Tu+H+/O+coXrkq7OtmNWms8pb9mzhzbhr8uson19ygcvAz7CfXu9dzjW/1A/2S7XNvlF+wH/DjDoH3qjnuMe3eGaDzjT+6k5AQRQsWqTxmyJHOnJsG6j+v4B5el9Zeqg8I5jjZanVv15NrdT/RN3/p5hyT//MElZP9vYg7OQAAwEsUOQAAwEsUOQAAwEte9OTsaFLHGetRoyDsMXPebK9yI/vDbmYiFeS94fYoHN32NpV/6/+EynP6PKPy9OOLnHPcNPe8sK/7asdXVQ5+1i7i7tcT/Az7luX6uTWzru/knMNs+i3sOpAcqq91OxTmF+nesDZZNcKeY5t198CpuZyfR1NF/efGO2P/17+HytfsMVbljtlV8yzFYL/fy4NPUrnhsODap8V5RfHH3xwAAOAlihwAAOAlihwAAOAlihwAAOAlLxqPy+OaJUep3Pj12SoXV+ViUCXqz9J/qs+sb61yp+pLVO5e3X0w5ued343wKm6jcdAzG1qo/PhHfVRud+9klc12moxTVe7b7qai5+6lG+B/vesplf+5uoPK7w47zjlHk6H8YkQqm99tu8p3tr1Af/3yvVQ+4cSfnXM8urf+5YrOL12nsinHN7E2r61RueEMt0naN9zJAQAAXqLIAQAAXqLIAQAAXjLWug823KlXxjm7/yJS2uclb7sNKHGSrNdRVsvmKs/9d92Ixzx4wHsq/7CprcqjxxziHNPqbn8/966q6yhZryFUHu9FiIXdXUfcyQEAAF6iyAEAAF6iyAEAAF5Km31ygKCiRX+o3Or8P3Yz83+GSevAiH4gYyvxt/8GAFINd3IAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXjLU20WsAAACIOe7kAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL6VVkWOMscaYLcaYf5Vzfl9jzOay49rGe31IfhW4hnqWXUMlxpie8V4fUkMFrqOBZfOtMSYr3utD8uP7WfmkVZFTZn9r7T07gzHmFGPMtLI//B+MMZ12fs1aO9Jam5uYZSKJBa+h44wxvxhjNhpjFhhj+u38mrX2i7Jr6I+ErBTJLHgddTHGTDLGbC373y47v2atvU9EOidklUhm//8aMsY0NMZ8b4xZY4xZb4wZb4w5YufEdP1+lo5Fzv9njGknIq+KyDUiUldERovIB/ykhPIyxmSLyCgReVZE6ojIeSLymDFm/4QuDCnFGFNNRN4XkVdEpJ6IvCgi75eNA+WxWUT+IiJ7SOk19B8RGZ3u38/SusgRkRNE5Ftr7XfW2iIpvSiaiMgxiV0WUkh9EaktIi/bUhNFZKaIdAp/GKB0F5EsERlkrS2w1g4RESMixyV0VUgZ1trt1trZ1toSKb12iqW02Kmf2JUlVroXOabsn2DeJzHLQaqx1q4UkddF5ApjTKYx5jARaSEi3yV2ZUgxnUVkitVPTJ4ifESFKBljpojIdhH5QERGWGtXJXhJCZXuRc7nInKMMaZ72W3hu0WkmojUTOyykGJeF5H/E5ECEflWRO6x1i5O7JKQYnJFZENgbIOI5CVgLUhh1tr9pPTu8oXCD1vpXeRYa2eJyGUiMlRElotIQxGZISJLErkupA5jTAcReVNELpXSArmziNxujDk5oQtDqtkspd+YdlVbRDYlYC1IcWUfXb0uIneme39gWhc5IiLW2nestftYaxuIyH1S+lHDxAQvC6ljHxGZba0dY60tsdbOFpGPROSkBK8LqWW6iOxnjNn14/P9ysaBisoWkdaJXkQipX2RY4w5sKyXYg8p/Q2Z0WV3eIDymCwi7cp+jdwYY9qISB8R+S3B60JqGSuljaI3GGNyjDHXlY1/lbglIZUYYw41xhxpjKlmjKlhjLlDRBqJyE+JXlsipX2RIyKDRWS9iMwu+9+rErscpBJr7Xwp/bXNISKyUUTGici7IjIyketCarHW7hCR06X0Y8/1UnpNnV42DpRHjog8KSJrRGSpiPQWkZOttcsSuqoEM7qZ32/GmO1S2hw6xFp7bznmXyEij4tIdRHpZK1dEOclIslV4BrqIaVFT46I9LbWfh3nJSIFVOA6uk9EbpbS66iWtbY4zktEkuP7WfmkVZEDAADSBx9XAQAAL1HkAAAAL1HkAAAAL4V9cFevjHNo2PHU5yVvm8izYoPryF9VdR1xDfmL9yLEwu6uI+7kAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL1HkAAAAL2UlegFAKslq0Uzl9Yc0UXl5nx3OMf0PGKfyTfXmqLzPd1eoXLKolnOOtgN/03O2bg2/zr33csaKlq8IewyA5FDU40BnbE3nHJW37WlVtm23qHzH/p855+hbR78HfLpVn/O2YX1VbvzQD5EXm+S4kwMAALxEkQMAALxEkQMAALxETw4QxrLbDlf5nitfV/mM3FURz5ER+FmiREpUnnLkSH3Ake459t9+o8ot7gv/WXnOm8XOWNHRYQ9BLBjjDK3qf5jK/a9/T+V+dZZV+mWHbWis8nunHurMKVm0RGVb6PaPITE2XKz/vL769xBnTo7R365LxDpzdpUh7rVYaPUxPWro3r7vbnhU5cMzb3HO0fTB1OrT4U4OAADwEkUOAADwEkUOAADwUtr05GTs31Hl2TfXUPmSLj85x1xff4LKPR69TeW9BqXWZ5MIL7NTvjMWbQ/On8UFztjvRTVVLpZslQ+qpnsjMkP0dfx25WCVu23UPTp7P6qvxSPrz3fOMUZqh1gxKiUjU8XF9xziTJl6zdCwpyiwun9qWZF7DVUPXBJ7Zuprqm9t3W/Td+w7zjkGr2ur8pd99lG5aNEfYdeJ+Nl4+maVs02mMyfYg/NH0TaV71lyasTX+WlWa/06tfR7z3dHPK3y4afr/blERBY/pvfWsQXu9ZpMuJMDAAC8RJEDAAC8RJEDAAC8RJEDAAC85EXjscnJccZW9NMPOPvpTt24ualEN1wd+satzjm+6aIb9Y65eKLKswdFtUwkuVl35jpjwUbj4HVz7M9XqdxocHXnHJljfwn7uquv1pvF9RnwjTPn7oa/qlzsXvLKd2vbhBj9M/xBiNrS23SjcaQmYxGRAluk8v6v6Sby1rePd47J7NhO5Vl35ak87bhnVA5uHCcicmO9eXrgQx2/6N5K5eLVa5xzID5aXrVU5QGfujt3TlurH7pbT182UjzH/WWDoHxZG/brhzzzV5XnnPK0M6fLLder3PSB5P4FHO7kAAAAL1HkAAAAL1HkAAAAL6VkT05Gdd33MGvQfs6ceafoz8afWK8/03574Ikqt3krxOfg+bqvYUqbLirbU/QOXVlb3YciZn05yRlDcvrvUe7nz8GfAwb8rjfcanzGjEq/bsNn9bX31Sr3CZ13D/3VGQtn9qftnLGm9ORUmsnSb5nVjoi+b2Wf/+qehnYhenCCimfO1cdcqr9+VD/doPHQHcOcc3SvXqhysEfny7x99QH05FSZ4nXrVJ48/DBnTt35etO94jnhe/0qInNL5PsenXvPVnnDAzFfRkxxJwcAAHiJIgcAAHiJIgcAAHgpJXpyMmrqh9Etfa2FyvO66T0iREQeW6d7EsZcf4zKuV//GPF1g/sO1Fy3UeWbxo9VecQKd2+DDV9GfBkkiX2rZTtjwYfiTZyj9xLJl9j3LeRNc3tnvtuu+9AaTC9y5uzKus/4RAxkNm+q8sQDX9/NzP95Yr1+KGKHZ3T/hdvJF72Gw3Rfz6irDnLmdG8cufcHyaHBiOT9s+rTUD+081VpupuZyYE7OQAAwEsUOQAAwEsUOQAAwEtJ2ZMT7MGZ9eg+Kgd7cB5Z2945xzendlI5c2Hl9xRYfLnu8+lRY4zKa/dwX+OlunoPn+L1Gyq9DsTHsdPOcsY+3+ctlV/sPkLlf4neO6kiinro56ztcb/7DJrWWfq6aXjLQpW3vK/nG91KhBhZdF7jsF/fbAucsTce0Hty1ZkRuR+wshZc3tIZ+370TyofkVOi8tx++t+t9b36eUoiIrYofC8YUkvBSd1UvrzX2IjHvLeqa2Akufff4k4OAADwEkUOAADwEkUOAADwEkUOAADwUlI2Hv950f4qzzv1SZU/2pqr8jendXbOUbRwUczXtaNO+G7OmdvdpkQajVNH7k3uX4en39HN5v3qzFF5zlMHq9zpP8udc6w8Xm+Wdcp141S+tO5glRtn5YRYnR57qfVolfv01g99LKpB53EsZDaor/Idl721m5ml3tnUyhmr82r8G42DiqfPdsYuG9NP5Xmn6l/gmHmpfp89+d3AU0BFRH6eVvnFoUpk1q7tjK28QH+vvPom/RsLfWsvUXlR0TbnHGse1td4dRqPAQAAqh5FDgAA8BJFDgAA8FLCe3Kymrh9LLff9prKS4u3qvzgfQNUrr0gPp95Z7VuqXKfk34KPRFeKJ451xl7efBJKve/T8+ZdZruY5DT3PNmBH6WKJGSwIxQPTjaHSsOU3n0N/oBjB2m6s/Sr35ohnOOMfe6n9EjPFNdPxj1orxVCVpJ5dWeFXi7PzX8/NnXVHfG8q+M4YIQlYwueoPbZd3rqryxvd6o8aojdO+fiMhtDb6O8Cr6yb49P77ZmZE/ekKEcyQX7uQAAAAvUeQAAAAvUeQAAAAvJbwnp6SB2ydwVq11Kv9j9SEq136t8j04Jkv/qy+96WBnzp1Xvany+bnJvR8AKmfbae41cNTVE+P+un1/76Xynzc3d+ZkTJmnctut+u8Aj01MDl+v6xBidH2VrwPJLWvvvVS+bNx4lU+oucI5Jlt0L0y2yYz5uo68Vfe75r8Z//e/eONODgAA8BJFDgAA8BJFDgAA8FLCe3LK49Tak1X+sN+NKmdvjfycnrUn62dwfHj4Uyq3yXJ/9/+9LXofgrYfXKNy8NkvE9e2CPHKyyKuDYmx9gq998y5t3zmzLmp3pzASPQ/FwQ/O+/0pH7OVLN//RA4wu3hCO6sE0mGifYIhLLgypZRzZ/2RidnrJEE/3yR7mw93Yt6Rq21gRnVqm4xuzDBb6UlxQlZRyxxJwcAAHiJIgcAAHiJIgcAAHiJIgcAAHgp4Y3HJVNnO2P5b+kNieacq5uEJ9wXeChiBXy6rYHKp4/4izOn+UOTVO7QfqOeEHjA3dyJbuNxaxqPk0ZWi2Yq33v3iyqfVHOTc0zwYZpriwtUPnWKvm5e2ucF5xxts/UDOLO2R1xqpZVYfn6Jhe0tdiR6CfDRcr2x7CGTLlS5655LnUO+/WpflWusNM6cXW1r5P5Czj/OekPls3JXq9z77rEqfyzdnXPkvRGfB2LHC++EAADASxQ5AADASxQ5AADASwnvyRHrfm7Y9q/6M7+DZ12rcklv/QDPUNavylO55bv669U+1Q8eaxZiw67gyuyUWSr/c/U+Kl98wjjnHD/cnphNnSCS2b6tyg+OeUXl9tl6k74/inS/jYhI71duU7ntU7+rXH+p3iywz8v6WhURmXXcCH3MCYE+rccDD9qLwQZcI1870RlryqZ0QFIoXqe/h+1xqs5LQhzTSsaHGI3Oy0/oh10/8XxNlb/aVz+UetxV7dyTvBX796t44k4OAADwEkUOAADwEkUOAADwUuJ7csqh4bOBzyKfjXzMnnFYR2aD+ip3ral7hyZtbRWHV0VFzb0vV+VgD84X23Tf1t//dYNzjpbP62uvKMJrtr1ksjN21riTVR7T+W2VDx2gHzi759DK9840fYD+m6qwvHiryrX/iHSFpI5a8+gn9E3R8hUq5wZa926ZeKTKH3d4zznHoVddp7Lz/TnJcCcHAAB4iSIHAAB4iSIHAAB4KSV6cpKFbaI7fU6uuVnlG789zDkmX36O65qwey8c+lzYrz984yUq1/8oPp8tz/+0tR7QH2nLlQNGq/zBUP1cNSSvvAzd51VQO9OZU6OqFrOLzI7u/iYXXzUmqnO0eHGBM+ZPx1H8ZNar54zZHfoZaCVbtlTVcqLy6TddVX78fLe374xrv1b522erx3VNlcWdHAAA4CWKHAAA4CWKHAAA4CWKHAAA4CUaj6OwtFf9sF/PWp1dRStBeWQGHrGaEajpc9a4D+SMh5Yv6AbOVy5tpvIRNeap/FHDfOccxavXxH5hiChvemBDvBN0zDU5Kh92o37wr4jIzJdivarImrywzBm7ud7csMd0fFE/XLb1n+6/C1xZzZqq3On9pc6cD9/Xv5TSfGBiNus0Ofp6/eO2A1W+vbe7+V9Q82qrAyNNQ85LFtzJAQAAXqLIAQAAXqLIAQAAXqInJwoF9WzkSUgar6w5XOWujb9TedFf9fzWD3ZyzlHy64xKr8MWFau8obimyh2r6Z81Vp3h9uQ0GB7dRoWbzj/UGct748cQMxFOszcW6YGbw8/ft+YSZ2ym7BW7Be3Ggn/rno+3mjwWYpbuxxi+QfeGtX1c94YVF7H1X3lsOLiJyv9u9IEz5+4rv1f5wIb6zaf9iI2VXseCc+qqXFivxJlzf893VD43V/cGZYhR2T2DyFP3n61yHUnu9xXu5AAAAC9R5AAAAC9R5AAAAC/RkwNvffbFAXrgUt2TM+XIkSove9/dN+fRVT1U/uTbrs6cSEadOUjl9tn6IY6TC/TPGnu8+ptzjlCfjYdz9t8+c8bGvFE7yrPABh6kOHhdW5VvrKf7WC7I+8M5x79e6q1y+0e2qlwyZVbU69p8ziEqT774cZVrBPbvEXF7cD44S/esFf8Zfh8dhFZr6TaV/7l6H2fO3xpOU3n2mU+pnHFmsBfG7f90+2XC94gG55fnmFXF+to84v1bnDn570xWOdk7VbmTAwAAvESRAwAAvESRAwAAvERPTiVkGl0j1pueoIUgpLaD5qv803n62WKH5BSq3DSrhnOORwN76zx63nfOnEgyRL9uSaDD5pNN++mvb9Wfi1fE8JlHOGPNZWqlz5tuitdvUPnLPoF+iw91DPboiIjM7TFC5ZcP1vvm/OcNve9IKBed+ZXOdR5VuYbRey+F8sQrp6ncdGZinp/knR+nqPjNzYc5U46/S/dy/bfDmyoHn4EWqp8mKFKPzqub9nSOOTtXP9Os86cDVG4xSp+z3Uc/OedI9h6cIO7kAAAAL1HkAAAAL1HkAAAAL1HkAAAAL9F4XAnFVjeQ1pu5OUErQSjFK1ep/O8Tz1J59oA9VO7X40vnHDfVr/wDOvv+cazKE8fo5tXWI4MbyLkPeYxW83NoMo6HokX6z+q1wSfoCTe6xwSbkS/JW6HzVUMrsBLdaPzCxsYqv3v2Mc4RTWe6TaSIvawvJ7mDgbeWU0/RF8qyC3aoPOEovVmgiMjZs89XefWHTVU2gR1DG7/qbjL54v66+Tz/q5/dtXqGOzkAAMBLFDkAAMBLFDkAAMBL9ORUQnAzQCS34jl6c8C2N+n8ldRyjvlKusXglTeq1Fz0JmxFMXgFJEaD4eNV/uyFhs6cL1p2UXnWdXqTtiMP1n1f303oFPF1Owxbp3LJnIUq28LZEc+BxKk+eoLKrUfrr58v+uGpIiJZovvB9hL3YbC7Kg4xlvXV2nKtzyd8lwYAAF6iyAEAAF6iyAEAAF6iJ6cS5hfqfXEy17sPVgz1uSgAP9nCHc5Y8dwFKre7UeeVgfnt5MeIr8P7ClA+3MkBAABeosgBAABeosgBAABeoicnCi3/pvfEGPC3IwMz5gsAAEgO3MkBAABeosgBAABeosgBAABeosgBAABeosgBAABeosgBAABeosgBAABeosgBAABeMtbaRK8BAAAg5riTAwAAvESRAwAAvESRAwAAvESRAwAAvESRAwAAvESRAwAAvESRAwAAvESRAwAAvJRWRY4xxhpjthhj/lXO+X2NMZvLjmsb7/Uh+XENIRa4jlBZFbiGBpbNt8aYrHivL1mk1Y7HxhgrIu2stfN2GRsmIseISDsR+Yu19oXyHIf0FLwWjDH5IvKwiBwuIpkiMlFEbrDWzg53HNJbiOvoKBH5JDCtloicba19d3fHIX3t5vtZFxEZKSIdRWSmiPS11v66y9dbishCEcm21hZV6YITJK3u5OzGbyIyQER+SfRCkJLqisgHItJeRBqJyAQReT+hK0LKsdZ+a63N3fmPiPQRkc0i8mmCl4YUYYypJqXvPa+ISD0ReVFE3i8bT1tpX+RYa5+01n4pItsTvRakHmvtBGvtSGvtWmttoYg8LiLtjTENEr02pLTLROQda+2WRC8EKaO7iGSJyCBrbYG1doiIGBE5LqGrSrC0L3KAGDtaRFZYa9ckeiFITcaYmiJytpT+JA6UV2cRmWJ1D8qUsvG0RZEDxIgxpqmIPCkiNyd6LUhpZ4nIahEZl+iFIKXkisiGwNgGEclLwFqSBkUOEAPGmD1E5DMRecpa+3qi14OUdpmIvGTT6bdCEAubRaR2YKy2iGxKwFqSBkUOUEnGmHpSWuB8YK0t169zAqEYY5pJaW/FSwleClLPdBHZzxhjdhnbr2w8baV9kWOMqWaMqS6lDVrZxpjqxpi0/++C8jHG1BaRMSLyvbX2zkSvBynvEhH5wVo7P9ELQcoZKyLFInKDMSbHGHNd2fhXiVtS4vHNvPQn8G1Sus/JsLL/f3RCV4RUcoaIdBORK8o2a9v5T/NELwwp6VKh4RgVYK3dISKnS+k1tF5E/iIip5eNp610K3IKRGSSMeb+nQPW2u7WWhP4Z6yIiDHmCmPM+rLjShKzZCQZdQ1Za18su2Zq7brPibX2DxGuIeyW814kImKt7WCtHRmczHWEEEJ9P5tsrT3QWlvDWnuAtXbyzq8ZY+6T0n3hCkQkbfq90mrHYwAAkD7S7U4OAABIExQ5AADASxQ5AADAS2Eft94r4xwadjz1ecnbJvKs2OA68ldVXUdcQ/7ivQixsLvriDs5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADASxQ5AADAS1mJXkCizH3xAJVn9xzuzDnuugEq1xz1U1zXBABALGR2bu+MLTqzgcoH9Z6m8kstvlG50BZH/bo9ru2vco33JkR9jljiTg4AAPASRQ4AAPASRQ4AAPBS2vbkiDUqlkiJM2VpD53bjYrnglDVslq1UHnxGU2cOZvyi1Run79U5dHtP1A5/8NrnHM0HaN/lqg9eYXKdvNWlYv//NM5h8nSf1WX3XCwykU19Pzmj0xyzmELCpwxAH7YeOGhKp9851hnzqgGU8Oeo9Dq96pQ3xcjeXrQYJVvm32pM6d45tyoz1tR3MkBAABeosgBAABeosgBAABeSt+enHJo03GZyiYnR2V6HFLLipsOV/nn255QuSKfPwePmNPnGXdOn/DnfXPT3io/99cznDnLjtJ/VadeNtiZs6tTxl7ljJnvfw17DIDklVG9usrz/95V5emXDFW5Iu9nsZCfXU3lmTfWc+e4rYtxw50cAADgJYocAADgJYocAADgJYocAADgJRqPw/i4w3sqn5bbS+ViGo+TWmbbViq/eOPjgRmVv/xHbd5T5bNyV0d9jvPylus84ilnToYEN+nSJhfor2du2O6cIzFtiKlt5Q26WX3jQe5/12SQnaM3rZx25PMRj+nT5MB4LQeVZYwzFGw0nnrJkMCM2N+z6PTW9RHnzDj3ibBff/DYt52x5w/uowcmhN+ksDK4kwMAALxEkQMAALxEkQMAALxETw68tay33mSvY7XwNf1xU89zxmrdXzvsMdnL16s8snFdZ05BA7051oCH9GfUZ+SuCvsaoUzbYVW+7ZYBKtec9lPU54Rry6H64akzjxkedn6wd0okNpuyuT1Z4c8Z6quvbGxW6XUgPkqO0v02C/q5c2YcF+zBCe+dzXs5Y3/7Tm802uwDfV3VeH+Cym3lR+ccpmtnPXBu+HWEen8b0rqWynkTnCkxw50cAADgJYocAADgJYocAADgJXpy4K0jL5kU9uvLi7epvHJqI2dO5knhX6PRz7rfZuVBme46euo9ICrSgxP04cYuKtccRQ9OPLQbsFDlM/N0T8PCy5urXFDP7YYx1hmKWknDHSrP7Pls2PkdPh7gjHW8fV5gZF1ll4WKCuyDE+zBmXHcsKhPecrsU1UuuXcPZ07+9z9Hfd5Ux50cAADgJYocAADgJYocAADgJXpy4K2Pft5f5YdO+Vbl5lm5Ks+8cGj0L3KFjtnG7ckptMWBEf2zxepAb9BRb9/qnGPsOY+ofHdD3efT/dxrVc59y93fAtErXr9BDwRys/uXVMk6Np97qB7oqeO8Qv3sqo4Pr3XOUbyOHpxEyaheXeXgc6ii3QNHROSngmyV7XFLVTaic7riTg4AAPASRQ4AAPASRQ4AAPASRQ4AAPASjcfwVn5//dS3Axr0VXnqES+oHIsHKRaG2Pjtgy31VB68sIfKGYMbqtzmY7dp+KhaN6s865QnVV7WSzc3578VcalIIcv77Aj79YFL+qhcPGd+PJeDKNmObVSeekn0jcYdv7xa5TbD9PtVhvwa/cLSAHdyAACAlyhyAACAlyhyAACAl+jJQdpoPVD3NXTvfO1uZsZW3Z9XqFxjwcLAjGCO3r75i1UuqPQZkUzm9hihckng59NJE9qp3FbWxH1NKL+lPeqonBHh/sKoLfWdt1aHmgAAB7BJREFUsXZDC/XAhKnOnEQI/rsEN0QN1adojTsWL9zJAQAAXqLIAQAAXqLIAQAAXqInB2mjePpslXOnV83rFkWeElH7/PAP25s6p5nK+bJiNzORikrEBrLeI8WE6HtAYmQ1a+qMnXjheJUj7cl1x1fnOWP5EyaEmFn1ltyrc/DfJdiDc9miwNNkRaTeRzNUDj7COJa4kwMAALxEkQMAALxEkQMAALyUvj05gQ+xQ+1bEPx9f6AqFPY80Bkb036YyuMLslVu/9RWlWnRSF3bTjs4xOiksMcU19d7qCx4rYsz58AWf6h8096f63OI3rzkqueuc87R7J8/hF0HRNYe5fbk/LPRqLDH9Jp2rsodb5/lzIln38ruLHpzP2fsuS4vRHWO+c90cMbqbhwfYmZ8cCcHAAB4iSIHAAB4iSIHAAB4iSIHAAB4KX0bjwNPCAu1OVNwU6OZD7RROf/qtTFfFtJPRl6eyg8MG+bMCTbBf7NZN/PZyVW0s2Gay2y0p8qbDm+l8rb67s+NGWeujuo1Xuw8KMRoTthjZh3/TFSvISLS9/deKk/6tJPKLR/71Tkm/BZ2EBFZc+rWyJMCFi9poHL+xso/tDcWbt/vM2fsoJzwLdB9/zhW5QafznPmVGUTNXdyAACAlyhyAACAlyhyAACAl9K3J6ciqvGJNCovs0F9lTe/VkflrjmR+8OeG3eMyu3kp9gsDkrh8QepnHfvIpVHtR6qcqhNRSM9jNGVHXlKQLC/5s+bm0c+6McpKjYXvdEf73YVc3eXT52xUNfFrvL7/hyv5URl4ye67/TS2qE2oQz/7zLjuc4qN/iz6jb+C4U7OQAAwEsUOQAAwEsUOQAAwEv05ABVbHFfvcfNz/sMjnjMP1frB+V1fHylykWVXxZC+P0k/RY5pvUYlV/d1ETl9cU1nXO8v2x/lVd93cSZs6shfZ91xnrU0DuLdPvlApXr95kTOGJ92NdA/BTbWPRlxUdmXd3/N++ZFipP3+95lcuz7k5vXa9y2+GJ7cEJ4k4OAADwEkUOAADwEkUOAADwEj05QAwFn0MlIrLy1cYqv7v/w4EZ1VQasq6DBI156CiV6yz4sWILRFTqztTPuMv/+BqVO96me2GK129wzlFNfle5aSAH/XZhC2fs6Opzwx4DBBX2PNAZa3S/vo5GNR8ZmBH5vscX2/R7XPvh+hmOVflcqvLgTg4AAPASRQ4AAPASRQ4AAPASRQ4AAPASjcfw1oqbDle5Ws/VKj/a6S2VS0Js4hXJvxadrPLAVu85c9wHblZz5uzq6/MOcsbqTKfROBEaDhsfyPrriWqyzH6lfuRJSBlrrjxM5QYjot9Qb87zutG4RZM1zpzhzb+M+rxB139ymcrtZiT3w4G5kwMAALxEkQMAALxEkQMAALxETw68sOm8Q52xn297Iuwx2SZT5UJbGPXrftxB9+AEz1l6Xp03lGxXuccjt6m81/Qfol4HUldmoz1VbpwdfrNAEZGs7cnxwEe4HppyvDN26ZHPh5j5P536Tlf5570Pd+b0O/9jla+tO1/lbPOryoU2VMdY+Psawfev/Bevc+a0uyu5HsAZCXdyAACAlyhyAACAlyhyAACAl9KmJ8dk6X/VnFo7ErQSxMPyXkXOWImE71sI9spEml8ewXOGOu//reihcpPP/lQ52R5wh/jadHgrlc/I/SjELH4eTRVNh2U7Y+O76V6XQ3J0/5+zf801kfezCb5bVeT9LPgw4OGjdT9R67//4hwT4i0uqfE3BwAAeIkiBwAAeIkiBwAAeCltenIyWjVX+dfDn4t4THA/k8Yfp81/rqSX2UA/u+eCAyckaCXRe7zxtyp/PTpX5SeO7O4cU7RiZTyXhCSSEeJnz+B7UdZmOreSVdaXk5yxWwf2V/nbB4bEfR1LigqcsYdW9lJ58eXNVG41Q++Bk2r9N6FwJwcAAHiJIgcAAHiJIgcAAHiJIgcAAHgpfTpp165Xcd+XblD5oKNnOYcsebidyrnv/RT7daFCCju1UPm+PcdUyeueOONslVeOa6InGPeYOy96S+Xz8parfGyNzSo/kVOt4gtEygu1idtLG/ZVOfsLt7kVyavhp/phml2b3ajy5P6DY/6aZwy+3Rnb+7Hgw3/nxPx1kw13cgAAgJcocgAAgJcocgAAgJfSpieneM1alVvdpTc9WhPimBqSOhvMpZvs5brH6sjJFzlzvuv6athzLC/epnKvl29z5rQdtkTlnGW6n6ZZ4e9hX0NE5I1nuqr8Zs3DVN54YGOV8zb6/zk5ojN85hEqN5epCVoJKqJ45SqVm/1T51P/2S3mr7m3BPtv0hN3cgAAgJcocgAAgJcocgAAgJfSpicHfimet1Dl+n3cOadKdJ9zt5TxzlhRVGcIrfjPP8N+vebvi/X8GLwmUsfSnpHn5H6cG3kSAAd3cgAAgJcocgAAgJcocgAAgJfoyQGABMrYrh949uiafZw59Z93+8UARMadHAAA4CWKHAAA4CWKHAAA4CWKHAAA4CUajwEggdrc8qPK46RGglYC+Ic7OQAAwEsUOQAAwEsUOQAAwEvGWpvoNQAAAMQcd3IAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICXKHIAAICX/h9oV8flQrdIBwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 16 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(10,10))\n",
    "\n",
    "for idx in range(16):\n",
    "    plt.subplot(4, 4, idx+1)\n",
    "    plt.axis('off')\n",
    "    plt.title('[{}]'.format(mnist2.train.labels[idx]))\n",
    "    plt.imshow(mnist2.train.images[idx].reshape((28,28)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-704cd7d225f4>:24: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-6-704cd7d225f4>:28: arg_max (from tensorflow.python.ops.gen_math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.math.argmax` instead\n"
     ]
    }
   ],
   "source": [
    "learning_rate=tf.placeholder(tf.float32)\n",
    "x = tf.placeholder(tf.float32,[None,784],name='x')\n",
    "y = tf.placeholder(tf.float32,[None,10],name='y')\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "  return tf.truncated_normal(shape, stddev=0.1)\n",
    "\n",
    "L1_count = 1000\n",
    "W_1 = tf.Variable(initialize([784,L1_count]),name='W1')\n",
    "b_1 = tf.Variable(initialize([L1_count]),name='b1')\n",
    "logits_1 = tf.matmul(x,W_1)+b_1\n",
    "output_1 = tf.nn.relu(logits_1)\n",
    "\n",
    "L2_count = 10\n",
    "W_2 = tf.Variable(initialize([L1_count,L2_count]),name='W2')\n",
    "b_2 = tf.Variable(initialize([L2_count]),name='b2')\n",
    "logits_2 = tf.matmul(output_1,W_2)+b_2\n",
    "logits = logits_2\n",
    "\n",
    "REGULARIZATION_RATE = 0.001\n",
    "regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)\n",
    "regularization = regularizer(W_1)+regularizer(W_2)\n",
    "\n",
    "cross_entropy_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=logits))\n",
    "total_loss = cross_entropy_loss+regularization\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "correct_prediction = tf.equal(tf.arg_max(y,1),tf.arg_max(logits,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "node {\n",
       "  name: \"Placeholder\"\n",
       "  op: \"Placeholder\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        unknown_rank: true\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"x\"\n",
       "  op: \"Placeholder\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: -1\n",
       "        }\n",
       "        dim {\n",
       "          size: 784\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"y\"\n",
       "  op: \"Placeholder\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: -1\n",
       "        }\n",
       "        dim {\n",
       "          size: 10\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 2\n",
       "          }\n",
       "        }\n",
       "        tensor_content: \"\\020\\003\\000\\000\\350\\003\\000\\000\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/mean\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/stddev\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.10000000149011612\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/TruncatedNormal\"\n",
       "  op: \"TruncatedNormal\"\n",
       "  input: \"truncated_normal/shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed2\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"truncated_normal/TruncatedNormal\"\n",
       "  input: \"truncated_normal/stddev\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal\"\n",
       "  op: \"Add\"\n",
       "  input: \"truncated_normal/mul\"\n",
       "  input: \"truncated_normal/mean\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"W1\"\n",
       "  op: \"VariableV2\"\n",
       "  attr {\n",
       "    key: \"container\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: 784\n",
       "        }\n",
       "        dim {\n",
       "          size: 1000\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shared_name\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"W1/Assign\"\n",
       "  op: \"Assign\"\n",
       "  input: \"W1\"\n",
       "  input: \"truncated_normal\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@W1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"validate_shape\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"W1/read\"\n",
       "  op: \"Identity\"\n",
       "  input: \"W1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@W1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_1/shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 1000\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_1/mean\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_1/stddev\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.10000000149011612\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_1/TruncatedNormal\"\n",
       "  op: \"TruncatedNormal\"\n",
       "  input: \"truncated_normal_1/shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed2\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_1/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"truncated_normal_1/TruncatedNormal\"\n",
       "  input: \"truncated_normal_1/stddev\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_1\"\n",
       "  op: \"Add\"\n",
       "  input: \"truncated_normal_1/mul\"\n",
       "  input: \"truncated_normal_1/mean\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"b1\"\n",
       "  op: \"VariableV2\"\n",
       "  attr {\n",
       "    key: \"container\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: 1000\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shared_name\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"b1/Assign\"\n",
       "  op: \"Assign\"\n",
       "  input: \"b1\"\n",
       "  input: \"truncated_normal_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@b1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"validate_shape\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"b1/read\"\n",
       "  op: \"Identity\"\n",
       "  input: \"b1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@b1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"MatMul\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"x\"\n",
       "  input: \"W1/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"add\"\n",
       "  op: \"Add\"\n",
       "  input: \"MatMul\"\n",
       "  input: \"b1/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Relu\"\n",
       "  op: \"Relu\"\n",
       "  input: \"add\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_2/shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 2\n",
       "          }\n",
       "        }\n",
       "        tensor_content: \"\\350\\003\\000\\000\\n\\000\\000\\000\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_2/mean\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_2/stddev\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.10000000149011612\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_2/TruncatedNormal\"\n",
       "  op: \"TruncatedNormal\"\n",
       "  input: \"truncated_normal_2/shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed2\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_2/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"truncated_normal_2/TruncatedNormal\"\n",
       "  input: \"truncated_normal_2/stddev\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_2\"\n",
       "  op: \"Add\"\n",
       "  input: \"truncated_normal_2/mul\"\n",
       "  input: \"truncated_normal_2/mean\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"W2\"\n",
       "  op: \"VariableV2\"\n",
       "  attr {\n",
       "    key: \"container\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: 1000\n",
       "        }\n",
       "        dim {\n",
       "          size: 10\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shared_name\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"W2/Assign\"\n",
       "  op: \"Assign\"\n",
       "  input: \"W2\"\n",
       "  input: \"truncated_normal_2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@W2\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"validate_shape\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"W2/read\"\n",
       "  op: \"Identity\"\n",
       "  input: \"W2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@W2\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_3/shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 10\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_3/mean\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_3/stddev\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.10000000149011612\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_3/TruncatedNormal\"\n",
       "  op: \"TruncatedNormal\"\n",
       "  input: \"truncated_normal_3/shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed2\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_3/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"truncated_normal_3/TruncatedNormal\"\n",
       "  input: \"truncated_normal_3/stddev\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal_3\"\n",
       "  op: \"Add\"\n",
       "  input: \"truncated_normal_3/mul\"\n",
       "  input: \"truncated_normal_3/mean\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"b2\"\n",
       "  op: \"VariableV2\"\n",
       "  attr {\n",
       "    key: \"container\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: 10\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shared_name\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"b2/Assign\"\n",
       "  op: \"Assign\"\n",
       "  input: \"b2\"\n",
       "  input: \"truncated_normal_3\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@b2\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"validate_shape\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"b2/read\"\n",
       "  op: \"Identity\"\n",
       "  input: \"b2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@b2\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"MatMul_1\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"Relu\"\n",
       "  input: \"W2/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"add_1\"\n",
       "  op: \"Add\"\n",
       "  input: \"MatMul_1\"\n",
       "  input: \"b2/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"l2_regularizer/scale\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.0010000000474974513\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"l2_regularizer/L2Loss\"\n",
       "  op: \"L2Loss\"\n",
       "  input: \"W1/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"l2_regularizer\"\n",
       "  op: \"Mul\"\n",
       "  input: \"l2_regularizer/scale\"\n",
       "  input: \"l2_regularizer/L2Loss\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"l2_regularizer_1/scale\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.0010000000474974513\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"l2_regularizer_1/L2Loss\"\n",
       "  op: \"L2Loss\"\n",
       "  input: \"W2/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"l2_regularizer_1\"\n",
       "  op: \"Mul\"\n",
       "  input: \"l2_regularizer_1/scale\"\n",
       "  input: \"l2_regularizer_1/L2Loss\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"add_2\"\n",
       "  op: \"Add\"\n",
       "  input: \"l2_regularizer\"\n",
       "  input: \"l2_regularizer_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/labels_stop_gradient\"\n",
       "  op: \"StopGradient\"\n",
       "  input: \"y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Rank\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 2\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"add_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Rank_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 2\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Shape_1\"\n",
       "  op: \"Shape\"\n",
       "  input: \"add_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub\"\n",
       "  op: \"Sub\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Rank_1\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice/begin\"\n",
       "  op: \"Pack\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 1\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"axis\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice/size\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice\"\n",
       "  op: \"Slice\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Shape_1\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice/begin\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice/size\"\n",
       "  attr {\n",
       "    key: \"Index\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat/values_0\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: -1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat/axis\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat\"\n",
       "  op: \"ConcatV2\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat/values_0\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat/axis\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 2\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"add_1\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Rank_2\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 2\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Shape_2\"\n",
       "  op: \"Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/labels_stop_gradient\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub_1/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub_1\"\n",
       "  op: \"Sub\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Rank_2\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub_1/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_1/begin\"\n",
       "  op: \"Pack\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub_1\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 1\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"axis\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_1/size\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_1\"\n",
       "  op: \"Slice\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Shape_2\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_1/begin\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_1/size\"\n",
       "  attr {\n",
       "    key: \"Index\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat_1/values_0\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: -1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat_1/axis\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat_1\"\n",
       "  op: \"ConcatV2\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat_1/values_0\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_1\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat_1/axis\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 2\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Reshape_1\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/labels_stop_gradient\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg\"\n",
       "  op: \"SoftmaxCrossEntropyWithLogits\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub_2/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub_2\"\n",
       "  op: \"Sub\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Rank\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub_2/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_2/begin\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_2/size\"\n",
       "  op: \"Pack\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub_2\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 1\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"axis\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_2\"\n",
       "  op: \"Slice\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_2/begin\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_2/size\"\n",
       "  attr {\n",
       "    key: \"Index\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Reshape_2\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Const\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Mean\"\n",
       "  op: \"Mean\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape_2\"\n",
       "  input: \"Const\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"add_3\"\n",
       "  op: \"Add\"\n",
       "  input: \"Mean\"\n",
       "  input: \"add_2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/grad_ys_0\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 1.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Fill\"\n",
       "  op: \"Fill\"\n",
       "  input: \"gradients/Shape\"\n",
       "  input: \"gradients/grad_ys_0\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"index_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_3_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/Fill\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_3_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/Fill\"\n",
       "  input: \"^gradients/add_3_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/Fill\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_3_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/Fill\"\n",
       "  input: \"^gradients/add_3_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/Fill\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Reshape/shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/add_3_grad/tuple/control_dependency\"\n",
       "  input: \"gradients/Mean_grad/Reshape/shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape_2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Tile\"\n",
       "  op: \"Tile\"\n",
       "  input: \"gradients/Mean_grad/Reshape\"\n",
       "  input: \"gradients/Mean_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tmultiples\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Shape_1\"\n",
       "  op: \"Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape_2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Shape_2\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Const\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Prod\"\n",
       "  op: \"Prod\"\n",
       "  input: \"gradients/Mean_grad/Shape_1\"\n",
       "  input: \"gradients/Mean_grad/Const\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Const_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Prod_1\"\n",
       "  op: \"Prod\"\n",
       "  input: \"gradients/Mean_grad/Shape_2\"\n",
       "  input: \"gradients/Mean_grad/Const_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Maximum/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Maximum\"\n",
       "  op: \"Maximum\"\n",
       "  input: \"gradients/Mean_grad/Prod_1\"\n",
       "  input: \"gradients/Mean_grad/Maximum/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/floordiv\"\n",
       "  op: \"FloorDiv\"\n",
       "  input: \"gradients/Mean_grad/Prod\"\n",
       "  input: \"gradients/Mean_grad/Maximum\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Cast\"\n",
       "  op: \"Cast\"\n",
       "  input: \"gradients/Mean_grad/floordiv\"\n",
       "  attr {\n",
       "    key: \"DstT\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"SrcT\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Truncate\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/truediv\"\n",
       "  op: \"RealDiv\"\n",
       "  input: \"gradients/Mean_grad/Tile\"\n",
       "  input: \"gradients/Mean_grad/Cast\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_2_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/add_3_grad/tuple/control_dependency_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_2_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/add_3_grad/tuple/control_dependency_1\"\n",
       "  input: \"^gradients/add_2_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/Fill\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_2_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/add_3_grad/tuple/control_dependency_1\"\n",
       "  input: \"^gradients/add_2_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/Fill\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/Mean_grad/truediv\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_grad/Mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"gradients/add_2_grad/tuple/control_dependency\"\n",
       "  input: \"l2_regularizer/L2Loss\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_grad/Mul_1\"\n",
       "  op: \"Mul\"\n",
       "  input: \"gradients/add_2_grad/tuple/control_dependency\"\n",
       "  input: \"l2_regularizer/scale\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/l2_regularizer_grad/Mul\"\n",
       "  input: \"^gradients/l2_regularizer_grad/Mul_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/l2_regularizer_grad/Mul\"\n",
       "  input: \"^gradients/l2_regularizer_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/l2_regularizer_grad/Mul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/l2_regularizer_grad/Mul_1\"\n",
       "  input: \"^gradients/l2_regularizer_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/l2_regularizer_grad/Mul_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_1_grad/Mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"gradients/add_2_grad/tuple/control_dependency_1\"\n",
       "  input: \"l2_regularizer_1/L2Loss\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_1_grad/Mul_1\"\n",
       "  op: \"Mul\"\n",
       "  input: \"gradients/add_2_grad/tuple/control_dependency_1\"\n",
       "  input: \"l2_regularizer_1/scale\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_1_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/l2_regularizer_1_grad/Mul\"\n",
       "  input: \"^gradients/l2_regularizer_1_grad/Mul_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_1_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/l2_regularizer_1_grad/Mul\"\n",
       "  input: \"^gradients/l2_regularizer_1_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/l2_regularizer_1_grad/Mul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_1_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/l2_regularizer_1_grad/Mul_1\"\n",
       "  input: \"^gradients/l2_regularizer_1_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/l2_regularizer_1_grad/Mul_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/zeros_like\"\n",
       "  op: \"ZerosLike\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg:1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims/dim\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: -1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims\"\n",
       "  op: \"ExpandDims\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Reshape\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims/dim\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tdim\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg:1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/LogSoftmax\"\n",
       "  op: \"LogSoftmax\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/Neg\"\n",
       "  op: \"Neg\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/LogSoftmax\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims_1/dim\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: -1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims_1\"\n",
       "  op: \"ExpandDims\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Reshape\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims_1/dim\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tdim\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/mul_1\"\n",
       "  op: \"Mul\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims_1\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/Neg\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/softmax_cross_entropy_with_logits_sg_grad/mul\"\n",
       "  input: \"^gradients/softmax_cross_entropy_with_logits_sg_grad/mul_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/mul\"\n",
       "  input: \"^gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/softmax_cross_entropy_with_logits_sg_grad/mul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/mul_1\"\n",
       "  input: \"^gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/softmax_cross_entropy_with_logits_sg_grad/mul_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer/L2Loss_grad/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"W1/read\"\n",
       "  input: \"gradients/l2_regularizer_grad/tuple/control_dependency_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/l2_regularizer_1/L2Loss_grad/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"W2/read\"\n",
       "  input: \"gradients/l2_regularizer_1_grad/tuple/control_dependency_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"add_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/control_dependency\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"MatMul_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/Shape_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 10\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/BroadcastGradientArgs\"\n",
       "  op: \"BroadcastGradientArgs\"\n",
       "  input: \"gradients/add_1_grad/Shape\"\n",
       "  input: \"gradients/add_1_grad/Shape_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/Sum\"\n",
       "  op: \"Sum\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Reshape\"\n",
       "  input: \"gradients/add_1_grad/BroadcastGradientArgs\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/add_1_grad/Sum\"\n",
       "  input: \"gradients/add_1_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/Sum_1\"\n",
       "  op: \"Sum\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Reshape\"\n",
       "  input: \"gradients/add_1_grad/BroadcastGradientArgs:1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/Reshape_1\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/add_1_grad/Sum_1\"\n",
       "  input: \"gradients/add_1_grad/Shape_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/add_1_grad/Reshape\"\n",
       "  input: \"^gradients/add_1_grad/Reshape_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/add_1_grad/Reshape\"\n",
       "  input: \"^gradients/add_1_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/add_1_grad/Reshape\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_1_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/add_1_grad/Reshape_1\"\n",
       "  input: \"^gradients/add_1_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/add_1_grad/Reshape_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_1_grad/MatMul\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"gradients/add_1_grad/tuple/control_dependency\"\n",
       "  input: \"W2/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_1_grad/MatMul_1\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"Relu\"\n",
       "  input: \"gradients/add_1_grad/tuple/control_dependency\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_1_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/MatMul_1_grad/MatMul\"\n",
       "  input: \"^gradients/MatMul_1_grad/MatMul_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_1_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/MatMul_1_grad/MatMul\"\n",
       "  input: \"^gradients/MatMul_1_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/MatMul_1_grad/MatMul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_1_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/MatMul_1_grad/MatMul_1\"\n",
       "  input: \"^gradients/MatMul_1_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/MatMul_1_grad/MatMul_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Relu_grad/ReluGrad\"\n",
       "  op: \"ReluGrad\"\n",
       "  input: \"gradients/MatMul_1_grad/tuple/control_dependency\"\n",
       "  input: \"Relu\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/AddN\"\n",
       "  op: \"AddN\"\n",
       "  input: \"gradients/l2_regularizer_1/L2Loss_grad/mul\"\n",
       "  input: \"gradients/MatMul_1_grad/tuple/control_dependency_1\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 2\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/l2_regularizer_1/L2Loss_grad/mul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"MatMul\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Shape_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 1000\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/BroadcastGradientArgs\"\n",
       "  op: \"BroadcastGradientArgs\"\n",
       "  input: \"gradients/add_grad/Shape\"\n",
       "  input: \"gradients/add_grad/Shape_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Sum\"\n",
       "  op: \"Sum\"\n",
       "  input: \"gradients/Relu_grad/ReluGrad\"\n",
       "  input: \"gradients/add_grad/BroadcastGradientArgs\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/add_grad/Sum\"\n",
       "  input: \"gradients/add_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Sum_1\"\n",
       "  op: \"Sum\"\n",
       "  input: \"gradients/Relu_grad/ReluGrad\"\n",
       "  input: \"gradients/add_grad/BroadcastGradientArgs:1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Reshape_1\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/add_grad/Sum_1\"\n",
       "  input: \"gradients/add_grad/Shape_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/add_grad/Reshape\"\n",
       "  input: \"^gradients/add_grad/Reshape_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/add_grad/Reshape\"\n",
       "  input: \"^gradients/add_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/add_grad/Reshape\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/add_grad/Reshape_1\"\n",
       "  input: \"^gradients/add_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/add_grad/Reshape_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/MatMul\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"gradients/add_grad/tuple/control_dependency\"\n",
       "  input: \"W1/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/MatMul_1\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"x\"\n",
       "  input: \"gradients/add_grad/tuple/control_dependency\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/MatMul_grad/MatMul\"\n",
       "  input: \"^gradients/MatMul_grad/MatMul_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/MatMul_grad/MatMul\"\n",
       "  input: \"^gradients/MatMul_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/MatMul_grad/MatMul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/MatMul_grad/MatMul_1\"\n",
       "  input: \"^gradients/MatMul_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/MatMul_grad/MatMul_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/AddN_1\"\n",
       "  op: \"AddN\"\n",
       "  input: \"gradients/l2_regularizer/L2Loss_grad/mul\"\n",
       "  input: \"gradients/MatMul_grad/tuple/control_dependency_1\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 2\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/l2_regularizer/L2Loss_grad/mul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"GradientDescent/update_W1/ApplyGradientDescent\"\n",
       "  op: \"ApplyGradientDescent\"\n",
       "  input: \"W1\"\n",
       "  input: \"Placeholder\"\n",
       "  input: \"gradients/AddN_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@W1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"GradientDescent/update_b1/ApplyGradientDescent\"\n",
       "  op: \"ApplyGradientDescent\"\n",
       "  input: \"b1\"\n",
       "  input: \"Placeholder\"\n",
       "  input: \"gradients/add_grad/tuple/control_dependency_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@b1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"GradientDescent/update_W2/ApplyGradientDescent\"\n",
       "  op: \"ApplyGradientDescent\"\n",
       "  input: \"W2\"\n",
       "  input: \"Placeholder\"\n",
       "  input: \"gradients/AddN\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@W2\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"GradientDescent/update_b2/ApplyGradientDescent\"\n",
       "  op: \"ApplyGradientDescent\"\n",
       "  input: \"b2\"\n",
       "  input: \"Placeholder\"\n",
       "  input: \"gradients/add_1_grad/tuple/control_dependency_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@b2\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"GradientDescent\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^GradientDescent/update_W1/ApplyGradientDescent\"\n",
       "  input: \"^GradientDescent/update_W2/ApplyGradientDescent\"\n",
       "  input: \"^GradientDescent/update_b1/ApplyGradientDescent\"\n",
       "  input: \"^GradientDescent/update_b2/ApplyGradientDescent\"\n",
       "}\n",
       "node {\n",
       "  name: \"ArgMax/dimension\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"ArgMax\"\n",
       "  op: \"ArgMax\"\n",
       "  input: \"y\"\n",
       "  input: \"ArgMax/dimension\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"output_type\"\n",
       "    value {\n",
       "      type: DT_INT64\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"ArgMax_1/dimension\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"ArgMax_1\"\n",
       "  op: \"ArgMax\"\n",
       "  input: \"add_1\"\n",
       "  input: \"ArgMax_1/dimension\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"output_type\"\n",
       "    value {\n",
       "      type: DT_INT64\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Equal\"\n",
       "  op: \"Equal\"\n",
       "  input: \"ArgMax\"\n",
       "  input: \"ArgMax_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT64\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Cast\"\n",
       "  op: \"Cast\"\n",
       "  input: \"Equal\"\n",
       "  attr {\n",
       "    key: \"DstT\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"SrcT\"\n",
       "    value {\n",
       "      type: DT_BOOL\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Truncate\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Const_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Mean_1\"\n",
       "  op: \"Mean\"\n",
       "  input: \"Cast\"\n",
       "  input: \"Const_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "versions {\n",
       "  producer: 38\n",
       "}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grap = tf.get_default_graph()\n",
    "grap.as_graph_def()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "##########\n",
      "learning rate is 0.49\n",
      "after [1000] training steps, the loss is [0.09799337387084961], the validation accuracy is [0.9733999967575073]\n",
      "##########\n",
      "learning rate is 0.48019999999999996\n",
      "after [2000] training steps, the loss is [0.08723492920398712], the validation accuracy is [0.9783999919891357]\n",
      "##########\n",
      "learning rate is 0.47059599999999996\n",
      "after [3000] training steps, the loss is [0.06690418720245361], the validation accuracy is [0.9787999987602234]\n",
      "##########\n",
      "learning rate is 0.46118407999999994\n",
      "after [4000] training steps, the loss is [0.058914799243211746], the validation accuracy is [0.9797999858856201]\n",
      "##########\n",
      "learning rate is 0.4519603983999999\n",
      "after [5000] training steps, the loss is [0.061983928084373474], the validation accuracy is [0.9797999858856201]\n",
      "##########\n",
      "learning rate is 0.4429211904319999\n",
      "after [6000] training steps, the loss is [0.05641014128923416], the validation accuracy is [0.9797999858856201]\n",
      "##########\n",
      "learning rate is 0.4340627666233599\n",
      "after [7000] training steps, the loss is [0.06385537981987], the validation accuracy is [0.9801999926567078]\n",
      "##########\n",
      "learning rate is 0.4253815112908927\n",
      "after [8000] training steps, the loss is [0.053264960646629333], the validation accuracy is [0.9797999858856201]\n",
      "##########\n",
      "learning rate is 0.41687388106507484\n",
      "after [9000] training steps, the loss is [0.051856573671102524], the validation accuracy is [0.9801999926567078]\n",
      "##########\n",
      "learning rate is 0.40853640344377334\n",
      "after [10000] training steps, the loss is [0.038214899599552155], the validation accuracy is [0.9800000190734863]\n",
      "##########\n",
      "learning rate is 0.40036567537489787\n",
      "after [11000] training steps, the loss is [0.04779862239956856], the validation accuracy is [0.980400025844574]\n",
      "##########\n",
      "learning rate is 0.3923583618673999\n",
      "after [12000] training steps, the loss is [0.06113794073462486], the validation accuracy is [0.9801999926567078]\n",
      "##########\n",
      "learning rate is 0.3845111946300519\n",
      "after [13000] training steps, the loss is [0.04669136554002762], the validation accuracy is [0.9805999994277954]\n",
      "##########\n",
      "learning rate is 0.37682097073745086\n",
      "after [14000] training steps, the loss is [0.048445239663124084], the validation accuracy is [0.9814000129699707]\n",
      "##########\n",
      "learning rate is 0.3692845513227018\n",
      "after [15000] training steps, the loss is [0.05056079477071762], the validation accuracy is [0.9805999994277954]\n",
      "##########\n",
      "learning rate is 0.3618988602962478\n",
      "after [16000] training steps, the loss is [0.05566611513495445], the validation accuracy is [0.9805999994277954]\n",
      "##########\n",
      "learning rate is 0.35466088309032284\n",
      "after [17000] training steps, the loss is [0.06466574966907501], the validation accuracy is [0.9818000197410583]\n",
      "##########\n",
      "learning rate is 0.34756766542851636\n",
      "after [18000] training steps, the loss is [0.04756675660610199], the validation accuracy is [0.9805999994277954]\n",
      "##########\n",
      "learning rate is 0.34061631211994603\n",
      "after [19000] training steps, the loss is [0.042610470205545425], the validation accuracy is [0.9811999797821045]\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.981\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    lr = 0.5\n",
    "    for step in range(20000):\n",
    "        batch_x,batch_y = mnist.train.next_batch(512)\n",
    "        _,loss = sess.run(\n",
    "            [optimizer,cross_entropy_loss],\n",
    "            feed_dict={\n",
    "                x:batch_x,\n",
    "                y:batch_y,\n",
    "                learning_rate:lr\n",
    "            })\n",
    "        if step > 0 and step % 1000 == 0:\n",
    "            lr = lr*0.98;\n",
    "            print('#'*10);\n",
    "            print('learning rate is {}'.format(lr));\n",
    "            validate_accuracy = sess.run(accuracy,feed_dict={x:mnist.validation.images,y:mnist.validation.labels});\n",
    "            print('after [{}] training steps, the loss is [{}], the validation accuracy is [{}]'.format(step,loss,validate_accuracy))\n",
    "    print(\"the training is finish!\")\n",
    "    #最终的测试准确率\n",
    "    test_accuracy = sess.run(accuracy, feed_dict={x:mnist.test.images,y:mnist.test.labels})\n",
    "    print(\"the test accuarcy is:\", test_accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf1.14]",
   "language": "python",
   "name": "conda-env-tf1.14-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
